<!doctype html>



  


<html class="theme-next pisces use-motion" lang="zh-Hans">
<head>
  <meta charset="UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"/>



<meta http-equiv="Cache-Control" content="no-transform" />
<meta http-equiv="Cache-Control" content="no-siteapp" />












  
  
  <link href="/lib/fancybox/source/jquery.fancybox.css?v=2.1.5" rel="stylesheet" type="text/css" />







<link href="/lib/font-awesome/css/font-awesome.min.css?v=4.6.2" rel="stylesheet" type="text/css" />

<link href="/css/main.css?v=5.1.0" rel="stylesheet" type="text/css" />


  <meta name="keywords" content="学习,Scrapy," />








  <link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico?v=5.1.0" />






<meta name="description" content="前言
Scrapy学习（三） 爬取豆瓣图书信息

接上篇之后。这次来爬取需要登录才能访问的微博。爬虫目标是获取用户的微博数、关注数、粉丝数。为建立用户关系图(尚未实现)做数据储备">
<meta property="og:type" content="article">
<meta property="og:title" content="Scrapy学习（四） 爬取微博数据">
<meta property="og:url" content="http://yoursite.com/2017/02/13/Scrapy学习（四）-爬取微博数据/index.html">
<meta property="og:site_name" content="四畳半神话大系">
<meta property="og:description" content="前言
Scrapy学习（三） 爬取豆瓣图书信息

接上篇之后。这次来爬取需要登录才能访问的微博。爬虫目标是获取用户的微博数、关注数、粉丝数。为建立用户关系图(尚未实现)做数据储备">
<meta property="og:image" content="http://oi6538cys.bkt.clouddn.com/snipaste20170213_132630.png">
<meta property="og:image" content="http://oi6538cys.bkt.clouddn.com/snipaste20170213_141218.png">
<meta property="og:updated_time" content="2017-02-13T06:50:38.033Z">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="Scrapy学习（四） 爬取微博数据">
<meta name="twitter:description" content="前言
Scrapy学习（三） 爬取豆瓣图书信息

接上篇之后。这次来爬取需要登录才能访问的微博。爬虫目标是获取用户的微博数、关注数、粉丝数。为建立用户关系图(尚未实现)做数据储备">
<meta name="twitter:image" content="http://oi6538cys.bkt.clouddn.com/snipaste20170213_132630.png">



<script type="text/javascript" id="hexo.configurations">
  var NexT = window.NexT || {};
  var CONFIG = {
    root: '/',
    scheme: 'Pisces',
    sidebar: {"position":"left","display":"post"},
    fancybox: true,
    motion: true,
    duoshuo: {
      userId: '0',
      author: '博主'
    },
    algolia: {
      applicationID: '',
      apiKey: '',
      indexName: '',
      hits: {"per_page":10},
      labels: {"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}
    }
  };
</script>



  <link rel="canonical" href="http://yoursite.com/2017/02/13/Scrapy学习（四）-爬取微博数据/"/>





  <title> Scrapy学习（四） 爬取微博数据 | 四畳半神话大系 </title>
  <meta name="baidu-site-verification" content="UqNuoDzFTg" />
</head>

<body itemscope itemtype="http://schema.org/WebPage" lang="zh-Hans">

  



  <script type="text/javascript">
    var _hmt = _hmt || [];
    (function() {
      var hm = document.createElement("script");
      hm.src = "https://hm.baidu.com/hm.js?c3916ebb87080111f91966b1291316d7";
      var s = document.getElementsByTagName("script")[0];
      s.parentNode.insertBefore(hm, s);
    })();
  </script>








  
  
    
  

  <div class="container one-collumn sidebar-position-left page-post-detail ">
    <div class="headband"></div>

    <header id="header" class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-meta ">
  

  <div class="custom-logo-site-title">
    <a href="/"  class="brand" rel="start">
      <span class="logo-line-before"><i></i></span>
      <span class="site-title">四畳半神话大系</span>
      <span class="logo-line-after"><i></i></span>
    </a>
  </div>
  <p class="site-subtitle"></p>
</div>

<div class="site-nav-toggle">
  <button>
    <span class="btn-bar"></span>
    <span class="btn-bar"></span>
    <span class="btn-bar"></span>
  </button>
</div>

<nav class="site-nav">
  

  
    <ul id="menu" class="menu">
      
        
        <li class="menu-item menu-item-home">
          <a href="/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-home"></i> <br />
            
            首页
          </a>
        </li>
      
        
        <li class="menu-item menu-item-categories">
          <a href="/categories" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-th"></i> <br />
            
            分类
          </a>
        </li>
      
        
        <li class="menu-item menu-item-about">
          <a href="/about" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-user"></i> <br />
            
            关于
          </a>
        </li>
      
        
        <li class="menu-item menu-item-archives">
          <a href="/archives" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-archive"></i> <br />
            
            归档
          </a>
        </li>
      
        
        <li class="menu-item menu-item-tags">
          <a href="/tags" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-tags"></i> <br />
            
            标签
          </a>
        </li>
      
    </ul>
  

  
    <div class="site-search">
      
  <div class="popup">
 <span class="search-icon fa fa-search"></span>
 <input type="text" id="local-search-input">
 <div id="local-search-result"></div>
 <span class="popup-btn-close">close</span>
</div>


    </div>
  
</nav>



 </div>
    </header>

    <main id="main" class="main">
      <div class="main-inner">
        <div class="content-wrap">
        
        <div class="menu-item menu-item-search">
          
            <a href="javascript:;" class="popup-trigger"  style="margin-left: 70%;display: block;">
          
            
              <i class="menu-item-icon fa fa-search fa-fw"></i>搜索 <br />
            
          </a>
        </div>
        
          <div id="content" class="content">
            

  <div id="posts" class="posts-expand">
    

  

  
  
  

  <article class="post post-type-normal " itemscope itemtype="http://schema.org/Article" test>
  <link itemprop="mainEntityOfPage" href="http://yoursite.com/2017/02/13/Scrapy学习（四）-爬取微博数据/">

  <span style="display:none" itemprop="author" itemscope itemtype="http://schema.org/Person">
    <meta itemprop="name" content="Amoyiki">
    <meta itemprop="description" content="">
    <meta itemprop="image" content="/images/avatar.jpg">
  </span>

  <span style="display:none" itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
    <meta itemprop="name" content="四畳半神话大系">
    <span style="display:none" itemprop="logo" itemscope itemtype="http://schema.org/ImageObject">
      <img style="display:none;" itemprop="url image" alt="四畳半神话大系" src="">
    </span>
  </span>

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">
            
            
              
                Scrapy学习（四） 爬取微博数据
              
            
          </h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>
              <time title="Post created" itemprop="dateCreated datePublished" datetime="2017-02-13T14:48:00+08:00">
                2017-02-13
              </time>
            

            

            
          </span>

          
            <span class="post-category" >
              <span class="post-meta-divider">|</span>
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/Python/" itemprop="url" rel="index">
                    <span itemprop="name">Python</span>
                  </a>
                </span>

                
                
              
            </span>
          

          
            
              <span class="post-comments-count">
                <span class="post-meta-divider">|</span>
                <a href="/2017/02/13/Scrapy学习（四）-爬取微博数据/#comments" itemprop="discussionUrl">
                  <span class="post-comments-count ds-thread-count" data-thread-key="2017/02/13/Scrapy学习（四）-爬取微博数据/" itemprop="commentCount"></span>
                </a>
              </span>
            
          

          

          
          

          
            <span class="post-meta-divider">|</span>
            <span class="page-pv"><i class="fa fa-file-o"></i>
            <span class="busuanzi-value" id="busuanzi_value_page_pv" ></span>
            </span>
          

          

        </div>
      </header>
    


    <div class="post-body" itemprop="articleBody">

      
      

      
        <h2 id="前言"><a href="#前言" class="headerlink" title="前言"></a>前言</h2><blockquote>
<p><a href="https://segmentfault.com/a/1190000008253696?_ea=1616863" target="_blank" rel="external">Scrapy学习（三） 爬取豆瓣图书信息</a></p>
</blockquote>
<p>接上篇之后。这次来爬取需要登录才能访问的微博。<br>爬虫目标是获取用户的微博数、关注数、粉丝数。为建立用户关系图(尚未实现)做数据储备<br><a id="more"></a></p>
<h2 id="准备"><a href="#准备" class="headerlink" title="准备"></a>准备</h2><ul>
<li>安装第三方库<code>requests</code>和<code>pymongo</code></li>
<li>安装MongoDB</li>
<li>创建一个weibo爬虫项目</li>
</ul>
<p>如何创建Scrapy项目之前文章都已经提到了，直接进入主题。</p>
<h2 id="创建Items"><a href="#创建Items" class="headerlink" title="创建Items"></a>创建Items</h2><p>Item数据这部分我只需要个人信息，微博数，关注数、分数数这些基本信息就行。<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br></pre></td><td class="code"><pre><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">ProfileItem</span><span class="params">(Item)</span>:</span></span><br><span class="line">    <span class="string">"""</span><br><span class="line">    账号的微博数、关注数、粉丝数及详情</span><br><span class="line">    """</span></span><br><span class="line">    _id = Field()</span><br><span class="line">    nick_name = Field()</span><br><span class="line">    profile_pic = Field()</span><br><span class="line">    tweet_stats = Field()</span><br><span class="line">    following_stats = Field()</span><br><span class="line">    follower_stats = Field()</span><br><span class="line">    sex = Field()</span><br><span class="line">    location = Field()</span><br><span class="line">    birthday = Field()</span><br><span class="line">    bio = Field()</span><br><span class="line">    </span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">FollowingItem</span><span class="params">(Item)</span>:</span></span><br><span class="line">    <span class="string">"""</span><br><span class="line">    关注的微博账号</span><br><span class="line">    """</span></span><br><span class="line">    _id = Field()</span><br><span class="line">    relationship = Field()</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">FollowedItem</span><span class="params">(Item)</span>:</span></span><br><span class="line">    <span class="string">"""</span><br><span class="line">    粉丝的微博账号</span><br><span class="line">    """</span></span><br><span class="line">    _id = Field()</span><br><span class="line">    relationship = Field()</span><br></pre></td></tr></table></figure></p>
<h2 id="编写Spider"><a href="#编写Spider" class="headerlink" title="编写Spider"></a>编写Spider</h2><p>为了方便爬虫，我们选择登陆的入口是手机版的微博<code>http://weibo.cn/</code>。<br><img src="http://oi6538cys.bkt.clouddn.com/snipaste20170213_132630.png" alt=""><br>其中微博的uid可以通过访问用户资料页或者从关注的<code>href</code>属性中获取</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br></pre></td><td class="code"><pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">WeiboSpiderSpider</span><span class="params">(scrapy.Spider)</span>:</span></span><br><span class="line">    name = <span class="string">"weibo_spider"</span></span><br><span class="line">    allowed_domains = [<span class="string">"weibo.cn"</span>]</span><br><span class="line">    url = <span class="string">"http://weibo.cn/"</span></span><br><span class="line">    start_urls = [<span class="string">'2634877355'</span>,...] <span class="comment"># 爬取入口微博ID</span></span><br><span class="line">    task_set = set(start_urls) <span class="comment"># 待爬取集合</span></span><br><span class="line">    tasked_set = set() <span class="comment"># 已爬取集合</span></span><br><span class="line">    ...   </span><br><span class="line">    </span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">start_requests</span><span class="params">(self)</span>:</span></span><br><span class="line">        <span class="keyword">while</span> len(self.task_set) &gt; <span class="number">0</span> :</span><br><span class="line">            _id = self.task_set.pop()</span><br><span class="line">            <span class="keyword">if</span> _id <span class="keyword">in</span> self.tasked_set:</span><br><span class="line">                <span class="keyword">raise</span> CloseSpider(reason=<span class="string">"已存在该数据 %s "</span>% (_id) )</span><br><span class="line">            self.tasked_set.add(_id)</span><br><span class="line">            info_url = self.url + _id</span><br><span class="line">            info_item = ProfileItem()</span><br><span class="line">            following_url = info_url + <span class="string">"/follow"</span></span><br><span class="line">            following_item = FollowingItem()</span><br><span class="line">            following_item[<span class="string">"_id"</span>] = _id</span><br><span class="line">            following_item[<span class="string">"relationship"</span>] = []</span><br><span class="line">            follower_url = info_url + <span class="string">"/fans"</span></span><br><span class="line">            follower_item = FollowedItem()</span><br><span class="line">            follower_item[<span class="string">"_id"</span>] = _id</span><br><span class="line">            follower_item[<span class="string">"relationship"</span>] = []</span><br><span class="line">            <span class="keyword">yield</span> scrapy.Request(info_url, meta=&#123;<span class="string">"item"</span>:info_item&#125;, callback=self.account_parse)</span><br><span class="line">            <span class="keyword">yield</span> scrapy.Request(following_url, meta=&#123;<span class="string">"item"</span>:following_item&#125;, callback=self.relationship_parse)</span><br><span class="line">            <span class="keyword">yield</span> scrapy.Request(follower_url, meta=&#123;<span class="string">"item"</span>:follower_item&#125;, callback=self.relationship_parse)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">account_parse</span><span class="params">(self, response)</span>:</span></span><br><span class="line">        item = response.meta[<span class="string">"item"</span>]</span><br><span class="line">        sel = scrapy.Selector(response)</span><br><span class="line">        profile_url = sel.xpath(<span class="string">"//div[@class='ut']/a/@href"</span>).extract()[<span class="number">1</span>]</span><br><span class="line">        counts = sel.xpath(<span class="string">"//div[@class='u']/div[@class='tip2']"</span>).extract_first()</span><br><span class="line">        item[<span class="string">'_id'</span>] = re.findall(<span class="string">u'^/(\d+)/info'</span>,profile_url)[<span class="number">0</span>]</span><br><span class="line">        item[<span class="string">'tweet_stats'</span>] = re.findall(<span class="string">u'微博\[(\d+)\]'</span>, counts)[<span class="number">0</span>]</span><br><span class="line">        item[<span class="string">'following_stats'</span>] = re.findall(<span class="string">u'关注\[(\d+)\]'</span>, counts)[<span class="number">0</span>]</span><br><span class="line">        item[<span class="string">'follower_stats'</span>] = re.findall(<span class="string">u'粉丝\[(\d+)\]'</span>, counts)[<span class="number">0</span>]</span><br><span class="line">        <span class="keyword">if</span> int(item[<span class="string">'tweet_stats'</span>]) &lt; <span class="number">4500</span> <span class="keyword">and</span> int(item[<span class="string">'following_stats'</span>]) &gt; <span class="number">1000</span> <span class="keyword">and</span> int(item[<span class="string">'follower_stats'</span>]) &lt; <span class="number">500</span>:</span><br><span class="line">            <span class="keyword">raise</span> CloseSpider(<span class="string">"僵尸粉"</span>)</span><br><span class="line">        <span class="keyword">yield</span> scrapy.Request(<span class="string">"http://weibo.cn"</span>+profile_url, meta=&#123;<span class="string">"item"</span>: item&#125;,callback=self.profile_parse)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">profile_parse</span><span class="params">(self,response)</span>:</span></span><br><span class="line">        item = response.meta[<span class="string">'item'</span>]</span><br><span class="line">        sel = scrapy.Selector(response)</span><br><span class="line">        info = sel.xpath(<span class="string">"//div[@class='tip']/following-sibling::div[@class='c']"</span>).extract_first()</span><br><span class="line">        item[<span class="string">"profile_pic"</span>] = sel.xpath(<span class="string">"//div[@class='c']/img/@src"</span>).extract_first()</span><br><span class="line">        item[<span class="string">"nick_name"</span>] = re.findall(<span class="string">u'昵称:(.*?)&lt;br&gt;'</span>,info)[<span class="number">0</span>]</span><br><span class="line">        item[<span class="string">"sex"</span>] = re.findall(<span class="string">u'性别:(.*?)&lt;br&gt;'</span>,info) <span class="keyword">and</span> re.findall(<span class="string">u'性别:(.*?)&lt;br&gt;'</span>,info)[<span class="number">0</span>] <span class="keyword">or</span> <span class="string">''</span></span><br><span class="line">        item[<span class="string">"location"</span>] = re.findall(<span class="string">u'地区:(.*?)&lt;br&gt;'</span>,info) <span class="keyword">and</span> re.findall(<span class="string">u'地区:(.*?)&lt;br&gt;'</span>,info)[<span class="number">0</span>] <span class="keyword">or</span> <span class="string">''</span></span><br><span class="line">        item[<span class="string">"birthday"</span>] = re.findall(<span class="string">u'生日:(.*?)&lt;br&gt;'</span>,info) <span class="keyword">and</span> re.findall(<span class="string">u'生日:(.*?)&lt;br&gt;'</span>,info)[<span class="number">0</span>] <span class="keyword">or</span> <span class="string">''</span></span><br><span class="line">        item[<span class="string">"bio"</span>] = re.findall(<span class="string">u'简介:(.*?)&lt;br&gt;'</span>,info) <span class="keyword">and</span> re.findall(<span class="string">u'简介:(.*?)&lt;br&gt;'</span>,info)[<span class="number">0</span>] <span class="keyword">or</span> <span class="string">''</span></span><br><span class="line">        <span class="keyword">yield</span> item</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">relationship_parse</span><span class="params">(self, response)</span>:</span></span><br><span class="line">        item = response.meta[<span class="string">"item"</span>]</span><br><span class="line">        sel = scrapy.Selector(response)</span><br><span class="line">        uids = sel.xpath(<span class="string">"//table/tr/td[last()]/a[last()]/@href"</span>).extract()</span><br><span class="line">        new_uids = []</span><br><span class="line">        <span class="keyword">for</span> uid <span class="keyword">in</span> uids:</span><br><span class="line">            <span class="keyword">if</span> <span class="string">"uid"</span> <span class="keyword">in</span> uid:</span><br><span class="line">                new_uids.append(re.findall(<span class="string">'uid=(\d+)&amp;'</span>,uid)[<span class="number">0</span>])</span><br><span class="line">            <span class="keyword">else</span>:</span><br><span class="line">                <span class="keyword">try</span>:</span><br><span class="line">                    new_uids.append(re.findall(<span class="string">'/(\d+)'</span>, uid)[<span class="number">0</span>])</span><br><span class="line">                <span class="keyword">except</span>:</span><br><span class="line">                    print(<span class="string">'--------'</span>,uid)</span><br><span class="line">                    <span class="keyword">pass</span></span><br><span class="line">        item[<span class="string">"relationship"</span>].extend(new_uids)</span><br><span class="line">        <span class="keyword">for</span> i <span class="keyword">in</span> new_uids:</span><br><span class="line">            <span class="keyword">if</span> i <span class="keyword">not</span> <span class="keyword">in</span> self.tasked_set:</span><br><span class="line">                self.task_set.add(i)</span><br><span class="line">        next_page = sel.xpath(<span class="string">"//*[@id='pagelist']/form/div/a[text()='下页']/@href"</span>).extract_first()</span><br><span class="line">        <span class="keyword">if</span> next_page:</span><br><span class="line">            <span class="keyword">yield</span> scrapy.Request(<span class="string">"http://weibo.cn"</span>+next_page, meta=&#123;<span class="string">"item"</span>: item&#125;,callback=self.relationship_parse)</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            <span class="keyword">yield</span> item</span><br></pre></td></tr></table></figure>
<p>代码中值得注意的地方有几个。</p>
<h3 id="start-url"><a href="#start-url" class="headerlink" title="start_url"></a>start_url</h3><p>这里我们填写的是微博的uid，有的用户有自定义域名（如上图），要访问后才能得到真正的uid<br><code>start_url</code> 填写的初始种子数要在10个以上。这是为了确保后面我们爬取到的新的种子能够加入到待爬取的队列中。10个以上的规定是从<a href="https://doc.scrapy.org/en/latest/topics/settings.html#reactor-threadpool-maxsize" target="_blank" rel="external">Scrapy文档</a>中查得的</p>
<blockquote>
<p>####REACTOR_THREADPOOL_MAXSIZE###<br>Default: <code>10</code><br>线程数是<code>Twisted</code>线程池的默认大小(The maximum limit for Twisted Reactor thread pool size.)</p>
</blockquote>
<h3 id="CloseSpider"><a href="#CloseSpider" class="headerlink" title="CloseSpider"></a>CloseSpider</h3><p>当遇到不需要的继续爬取的连接时(如已经爬取过的链接，定义的僵尸粉链接等等)，就可以用<a href="https://doc.scrapy.org/en/latest/topics/exceptions.html#closespider" target="_blank" rel="external">CloseSpider</a>关闭当前爬虫线程</p>
<h2 id="编写middlewares"><a href="#编写middlewares" class="headerlink" title="编写middlewares"></a>编写middlewares</h2><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">CookiesMiddleware</span><span class="params">(object)</span>:</span></span><br><span class="line">    <span class="string">""" 换Cookie """</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_request</span><span class="params">(self, request, spider)</span>:</span></span><br><span class="line">        cookie = random.choice(cookies)</span><br><span class="line">        request.cookies = cookie</span><br></pre></td></tr></table></figure>
<h2 id="编写cookie的获取方法"><a href="#编写cookie的获取方法" class="headerlink" title="编写cookie的获取方法"></a>编写cookie的获取方法</h2><p>这里我原本是想用手机版的微博去模拟登陆的，奈何验证码是在是太难搞了。所以我直接用网上有人编写好的登陆网页版微博的代码<a href="https://github.com/LiuXingMing/SinaSpider/blob/master/Sina_spider1/Sina_spider1/cookies.py" target="_blank" rel="external">SinaSpider</a> 这位写的很好，有兴趣的可以去看看。其中还有另一位写了<a href="https://github.com/xchaoinfo/fuck-login/blob/master/007%20weibo.com/weibo.com.py" target="_blank" rel="external">模拟登陆（带验证码）</a> 经测试可用。只不过我还没想好怎么嵌入到我的项目中。<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># encoding=utf-8</span></span><br><span class="line"><span class="keyword">import</span> json</span><br><span class="line"><span class="keyword">import</span> base64</span><br><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"></span><br><span class="line">myWeiBo = [</span><br><span class="line">    &#123;<span class="string">'no'</span>: <span class="string">'xx@sina.com'</span>, <span class="string">'psw'</span>: <span class="string">'xx'</span>&#125;,</span><br><span class="line">    &#123;<span class="string">'no'</span>: <span class="string">'xx@qq.com'</span>, <span class="string">'psw'</span>: <span class="string">'xx'</span>&#125;,</span><br><span class="line">]</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">getCookies</span><span class="params">(weibo)</span>:</span></span><br><span class="line">    <span class="string">""" 获取Cookies """</span></span><br><span class="line">    cookies = []</span><br><span class="line">    loginURL = <span class="string">r'https://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.4.15)'</span></span><br><span class="line">    <span class="keyword">for</span> elem <span class="keyword">in</span> weibo:</span><br><span class="line">        account = elem[<span class="string">'no'</span>]</span><br><span class="line">        password = elem[<span class="string">'psw'</span>]</span><br><span class="line">        username = base64.b64encode(account.encode(<span class="string">'utf-8'</span>)).decode(<span class="string">'utf-8'</span>)</span><br><span class="line">        postData = &#123;</span><br><span class="line">            <span class="string">"entry"</span>: <span class="string">"sso"</span>,</span><br><span class="line">            <span class="string">"gateway"</span>: <span class="string">"1"</span>,</span><br><span class="line">            <span class="string">"from"</span>: <span class="string">"null"</span>,</span><br><span class="line">            <span class="string">"savestate"</span>: <span class="string">"30"</span>,</span><br><span class="line">            <span class="string">"useticket"</span>: <span class="string">"0"</span>,</span><br><span class="line">            <span class="string">"pagerefer"</span>: <span class="string">""</span>,</span><br><span class="line">            <span class="string">"vsnf"</span>: <span class="string">"1"</span>,</span><br><span class="line">            <span class="string">"su"</span>: username,</span><br><span class="line">            <span class="string">"service"</span>: <span class="string">"sso"</span>,</span><br><span class="line">            <span class="string">"sp"</span>: password,</span><br><span class="line">            <span class="string">"sr"</span>: <span class="string">"1440*900"</span>,</span><br><span class="line">            <span class="string">"encoding"</span>: <span class="string">"UTF-8"</span>,</span><br><span class="line">            <span class="string">"cdult"</span>: <span class="string">"3"</span>,</span><br><span class="line">            <span class="string">"domain"</span>: <span class="string">"sina.com.cn"</span>,</span><br><span class="line">            <span class="string">"prelt"</span>: <span class="string">"0"</span>,</span><br><span class="line">            <span class="string">"returntype"</span>: <span class="string">"TEXT"</span>,</span><br><span class="line">        &#125;</span><br><span class="line">        session = requests.Session()</span><br><span class="line">        r = session.post(loginURL, data=postData)</span><br><span class="line">        jsonStr = r.content.decode(<span class="string">'gbk'</span>)</span><br><span class="line">        info = json.loads(jsonStr)</span><br><span class="line">        <span class="keyword">if</span> info[<span class="string">"retcode"</span>] == <span class="string">"0"</span>:</span><br><span class="line">            print(<span class="string">"Get Cookie Success!( Account:%s )"</span> % account)</span><br><span class="line">            cookie = session.cookies.get_dict()</span><br><span class="line">            cookies.append(cookie)</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            print(<span class="string">"Failed!( Reason:%s )"</span> % info[<span class="string">"reason"</span>].encode(<span class="string">"utf-8"</span>))</span><br><span class="line">    <span class="keyword">return</span> cookies</span><br><span class="line"></span><br><span class="line">cookies = getCookies(myWeiBo)</span><br></pre></td></tr></table></figure></p>
<p>登陆-反爬虫的这部分应该是整个项目中最难的地方了。<del>好多地方我都还不太懂。以后有空在研究</del></p>
<h2 id="编写pipelines"><a href="#编写pipelines" class="headerlink" title="编写pipelines"></a>编写pipelines</h2><p>这边只需要主要什么类型的Item存到那张表里就行了<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br></pre></td><td class="code"><pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">MongoDBPipeline</span><span class="params">(object)</span>:</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(self)</span>:</span></span><br><span class="line">        connection = MongoClient(</span><br><span class="line">            host=settings[<span class="string">'MONGODB_SERVER'</span>],</span><br><span class="line">            port=settings[<span class="string">'MONGODB_PORT'</span>]</span><br><span class="line">        )</span><br><span class="line">        db = connection[settings[<span class="string">'MONGODB_DB'</span>]]</span><br><span class="line">        self.info = db[settings[<span class="string">'INFO'</span>]]</span><br><span class="line">        self.following = db[settings[<span class="string">'FOLLOWING'</span>]]</span><br><span class="line">        self.followed = db[settings[<span class="string">'FOLLOWED'</span>]]</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_item</span><span class="params">(self, item, spider)</span>:</span></span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> isinstance(item, ProfileItem):</span><br><span class="line">            self.info.insert(dict(item))</span><br><span class="line">        <span class="keyword">elif</span> isinstance(item, FollowingItem):</span><br><span class="line">            self.following.insert(dict(item))</span><br><span class="line">        <span class="keyword">elif</span> isinstance(item, FollowedItem):</span><br><span class="line">            self.followed.insert(dict(item))</span><br><span class="line">        log.msg(<span class="string">"Weibo  added to MongoDB database!"</span>,</span><br><span class="line">                level=log.DEBUG, spider=spider)</span><br><span class="line">        <span class="keyword">return</span> item</span><br></pre></td></tr></table></figure></p>
<p>运行一下程序，就能看到MongoDB中有了我们要的数据了<br><img src="http://oi6538cys.bkt.clouddn.com/snipaste20170213_141218.png" alt=""></p>
<h2 id="总结"><a href="#总结" class="headerlink" title="总结"></a>总结</h2><ul>
<li><code>settings</code>中的<code>DOWNLOAD_DELAY</code>设置5才能防止被微博BAN掉</li>
<li>尝试在利用cookies登陆失败时使用模拟登陆，但是效果很不理想</li>
<li>尝试用代理IP池反爬虫，但是尝试失败<del>主要是不太会</del></li>
<li>未来将利用D3.js将爬到的数据绘制出来(<del>或许吧</del>)</li>
</ul>
<p>项目地址：<a href="https://github.com/amoyiki/weibo_spider" target="_blank" rel="external">weibo_spider</a></p>

      
    </div>

    <div>
      
        

      
    </div>

    <div>
      
        

      
    </div>


    <footer class="post-footer">
      
        <div class="post-tags">
          
            <a href="/tags/学习/" rel="tag"># 学习</a>
          
            <a href="/tags/Scrapy/" rel="tag"># Scrapy</a>
          
        </div>
      

      
        <div class="post-nav">
          <div class="post-nav-next post-nav-item">
            
              <a href="/2017/02/04/Scrapy学习（三）-爬取豆瓣图书信息/" rel="next" title="Scrapy学习（三） 爬取豆瓣图书信息">
                <i class="fa fa-chevron-left"></i> Scrapy学习（三） 爬取豆瓣图书信息
              </a>
            
          </div>

          <span class="post-nav-divider"></span>

          <div class="post-nav-prev post-nav-item">
            
          </div>
        </div>
      

      
      
    </footer>
  </article>



    <div class="post-spread">
      
    </div>
  </div>


          </div>
          


          
  <div class="comments" id="comments">
    
      <div class="ds-thread" data-thread-key="2017/02/13/Scrapy学习（四）-爬取微博数据/"
           data-title="Scrapy学习（四） 爬取微博数据" data-url="http://yoursite.com/2017/02/13/Scrapy学习（四）-爬取微博数据/">
      </div>
    
  </div>


        </div>
        
          
  
  <div class="sidebar-toggle">
    <div class="sidebar-toggle-line-wrap">
      <span class="sidebar-toggle-line sidebar-toggle-line-first"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-middle"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-last"></span>
    </div>
  </div>

  <aside id="sidebar" class="sidebar">
    <div class="sidebar-inner">

      

      
        <ul class="sidebar-nav motion-element">
          <li class="sidebar-nav-toc sidebar-nav-active" data-target="post-toc-wrap" >
            文章目录
          </li>
          <li class="sidebar-nav-overview" data-target="site-overview">
            站点概览
          </li>
        </ul>
      

      <section class="site-overview sidebar-panel">
        <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
          <img class="site-author-image" itemprop="image"
               src="/images/avatar.jpg"
               alt="Amoyiki" />
          <p class="site-author-name" itemprop="name">Amoyiki</p>
          <p class="site-description motion-element" itemprop="description">个人BLOG</p>
        </div>
        <nav class="site-state motion-element">
          <div class="site-state-item site-state-posts">
            <a href="/archives">
              <span class="site-state-item-count">36</span>
              <span class="site-state-item-name">日志</span>
            </a>
          </div>

          
            <div class="site-state-item site-state-categories">
              <a href="/categories">
                <span class="site-state-item-count">12</span>
                <span class="site-state-item-name">分类</span>
              </a>
            </div>
          

          
            <div class="site-state-item site-state-tags">
              <a href="/tags">
                <span class="site-state-item-count">8</span>
                <span class="site-state-item-name">标签</span>
              </a>
            </div>
          

        </nav>

        

        <div class="links-of-author motion-element">
          
            
              <span class="links-of-author-item">
                <a href="http://www.zhihu.com/people/amoyiki" target="_blank" title="Zhihu">
                  
                    <i class="fa fa-fw fa-globe"></i>
                  
                  Zhihu
                </a>
              </span>
            
              <span class="links-of-author-item">
                <a href="http://weibo.com/amoyiki" target="_blank" title="Weibo">
                  
                    <i class="fa fa-fw fa-weibo"></i>
                  
                  Weibo
                </a>
              </span>
            
          
        </div>

        
        

        
        

        


      </section>

      
      <!--noindex-->
        <section class="post-toc-wrap motion-element sidebar-panel sidebar-panel-active">
          <div class="post-toc">

            
              
            

            
              <div class="post-toc-content"><ol class="nav"><li class="nav-item nav-level-2"><a class="nav-link" href="#前言"><span class="nav-number">1.</span> <span class="nav-text">前言</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#准备"><span class="nav-number">2.</span> <span class="nav-text">准备</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#创建Items"><span class="nav-number">3.</span> <span class="nav-text">创建Items</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#编写Spider"><span class="nav-number">4.</span> <span class="nav-text">编写Spider</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#start-url"><span class="nav-number">4.1.</span> <span class="nav-text">start_url</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#CloseSpider"><span class="nav-number">4.2.</span> <span class="nav-text">CloseSpider</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#编写middlewares"><span class="nav-number">5.</span> <span class="nav-text">编写middlewares</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#编写cookie的获取方法"><span class="nav-number">6.</span> <span class="nav-text">编写cookie的获取方法</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#编写pipelines"><span class="nav-number">7.</span> <span class="nav-text">编写pipelines</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#总结"><span class="nav-number">8.</span> <span class="nav-text">总结</span></a></li></ol></div>
            

          </div>
        </section>
      <!--/noindex-->
      

    </div>
  </aside>


        
      </div>
    </main>

    <footer id="footer" class="footer">
      <div class="footer-inner">
        <div class="copyright" >
  
  &copy;  2015 - 
  <span itemprop="copyrightYear">2017</span>
  <span class="with-love">
    <i class="fa fa-heart"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">Amoyiki</span>
</div>


<div class="powered-by">
  由 <a class="theme-link" href="https://hexo.io">Hexo</a> 强力驱动
</div>

<div class="theme-info">
  主题 -
  <a class="theme-link" href="https://github.com/iissnan/hexo-theme-next">
    NexT.Pisces
  </a>
</div>


        

<div class="busuanzi-count">

  <script async src="https://dn-lbstatics.qbox.me/busuanzi/2.3/busuanzi.pure.mini.js"></script>

  
    <span class="site-uv"><i class="fa fa-user"></i><span class="busuanzi-value" id="busuanzi_value_site_uv"></span></span>
  

  
  
</div>



        
      </div>
    </footer>

    <div class="back-to-top">
      <i class="fa fa-arrow-up"></i>
    </div>
  </div>

  

<script type="text/javascript">
  if (Object.prototype.toString.call(window.Promise) !== '[object Function]') {
    window.Promise = null;
  }
</script>









  



  
  <script type="text/javascript" src="/lib/jquery/index.js?v=2.1.3"></script>

  
  <script type="text/javascript" src="/lib/fastclick/lib/fastclick.min.js?v=1.0.6"></script>

  
  <script type="text/javascript" src="/lib/jquery_lazyload/jquery.lazyload.js?v=1.9.7"></script>

  
  <script type="text/javascript" src="/lib/velocity/velocity.min.js?v=1.2.1"></script>

  
  <script type="text/javascript" src="/lib/velocity/velocity.ui.min.js?v=1.2.1"></script>

  
  <script type="text/javascript" src="/lib/fancybox/source/jquery.fancybox.pack.js?v=2.1.5"></script>


  


  <script type="text/javascript" src="/js/src/utils.js?v=5.1.0"></script>

  <script type="text/javascript" src="/js/src/motion.js?v=5.1.0"></script>



  
  


  <script type="text/javascript" src="/js/src/affix.js?v=5.1.0"></script>

  <script type="text/javascript" src="/js/src/schemes/pisces.js?v=5.1.0"></script>



  
  <script type="text/javascript" src="/js/src/scrollspy.js?v=5.1.0"></script>
<script type="text/javascript" src="/js/src/post-details.js?v=5.1.0"></script>



  


  <script type="text/javascript" src="/js/src/bootstrap.js?v=5.1.0"></script>



  

  
    
  

  <script type="text/javascript">
    var duoshuoQuery = {short_name:"amoyiki"};
    (function() {
      var ds = document.createElement('script');
      ds.type = 'text/javascript';ds.async = true;
      ds.id = 'duoshuo-script';
      ds.src = (document.location.protocol == 'https:' ? 'https:' : 'http:') + '//static.duoshuo.com/embed.js';
      ds.charset = 'UTF-8';
      (document.getElementsByTagName('head')[0]
      || document.getElementsByTagName('body')[0]).appendChild(ds);
    })();
  </script>

  
    
    
    <script src="/lib/ua-parser-js/dist/ua-parser.min.js?v=0.7.9"></script>
    <script src="/js/src/hook-duoshuo.js"></script>
  








  
  
  <script type="text/javascript">
    // Popup Window;
    var isfetched = false;
    // Search DB path;
    var search_path = "search.xml";
    if (search_path.length == 0) {
      search_path = "search.xml";
    }
    var path = "/" + search_path;
    // monitor main search box;

    function proceedsearch() {
      $("body").append('<div class="popoverlay">').css('overflow', 'hidden');
      $('.popup').toggle();
    }
    // search function;
    var searchFunc = function(path, search_id, content_id) {
      'use strict';
      $.ajax({
        url: path,
        dataType: "xml",
        async: true,
        success: function( xmlResponse ) {
          // get the contents from search data
          isfetched = true;
          $('.popup').detach().appendTo('.header-inner');
          var datas = $( "entry", xmlResponse ).map(function() {
            return {
              title: $( "title", this ).text(),
              content: $("content",this).text(),
              url: $( "url" , this).text()
            };
          }).get();
          var $input = document.getElementById(search_id);
          var $resultContent = document.getElementById(content_id);
          $input.addEventListener('input', function(){
            var matchcounts = 0;
            var str='<ul class=\"search-result-list\">';
            var keywords = this.value.trim().toLowerCase().split(/[\s\-]+/);
            $resultContent.innerHTML = "";
            if (this.value.trim().length > 1) {
              // perform local searching
              datas.forEach(function(data) {
                var isMatch = false;
                var content_index = [];
                var data_title = data.title.trim().toLowerCase();
                var data_content = data.content.trim().replace(/<[^>]+>/g,"").toLowerCase();
                var data_url = decodeURIComponent(data.url);
                var index_title = -1;
                var index_content = -1;
                var first_occur = -1;
                // only match artiles with not empty titles and contents
                if(data_title != '') {
                  keywords.forEach(function(keyword, i) {
                    index_title = data_title.indexOf(keyword);
                    index_content = data_content.indexOf(keyword);
                    if( index_title >= 0 || index_content >= 0 ){
                      isMatch = true;
                      if (i == 0) {
                        first_occur = index_content;
                      }
                    }

                  });
                }
                // show search results
                if (isMatch) {
                  matchcounts += 1;
                  str += "<li><a href='"+ data_url +"' class='search-result-title'>"+ data_title +"</a>";
                  var content = data.content.trim().replace(/<[^>]+>/g,"");
                  if (first_occur >= 0) {
                    // cut out 100 characters
                    var start = first_occur - 20;
                    var end = first_occur + 80;
                    if(start < 0){
                      start = 0;
                    }
                    if(start == 0){
                      end = 50;
                    }
                    if(end > content.length){
                      end = content.length;
                    }
                    var match_content = content.substring(start, end);
                    // highlight all keywords
                    keywords.forEach(function(keyword){
                      var regS = new RegExp(keyword, "gi");
                      match_content = match_content.replace(regS, "<b class=\"search-keyword\">"+keyword+"</b>");
                    });

                    str += "<p class=\"search-result\">" + match_content +"...</p>"
                  }
                  str += "</li>";
                }
              })};
            str += "</ul>";
            if (matchcounts == 0) { str = '<div id="no-result"><i class="fa fa-frown-o fa-5x" /></div>' }
            if (keywords == "") { str = '<div id="no-result"><i class="fa fa-search fa-5x" /></div>' }
            $resultContent.innerHTML = str;
          });
          proceedsearch();
        }
      });}

    // handle and trigger popup window;
    $('.popup-trigger').click(function(e) {
      e.stopPropagation();
      if (isfetched == false) {
        searchFunc(path, 'local-search-input', 'local-search-result');
      } else {
        proceedsearch();
      };
    });

    $('.popup-btn-close').click(function(e){
      $('.popup').hide();
      $(".popoverlay").remove();
      $('body').css('overflow', '');
    });
    $('.popup').click(function(e){
      e.stopPropagation();
    });
  </script>


  

  

  
<script>
(function(){
    var bp = document.createElement('script');
    var curProtocol = window.location.protocol.split(':')[0];
    if (curProtocol === 'https') {
        bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';        
    }
    else {
        bp.src = 'http://push.zhanzhang.baidu.com/push.js';
    }
    var s = document.getElementsByTagName("script")[0];
    s.parentNode.insertBefore(bp, s);
})();
</script>


  


</body>
</html>
